package com.fashion.schedule;

import com.fashion.exceptions.CollectException;
import com.fashion.model.CollectedNews;
import com.fashion.model.NewsSource;
import com.fashion.service.CollectedNewsService;
import com.fashion.service.NewsService;
import com.fashion.service.NewsSourceService;
import com.fashion.utils.StringUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;

/**
 * Created with IntelliJ IDEA.
 * User: gnyk
 * Date: 13-2-23
 */
public class ScheduledNewsCollector {
    private NewsSourceService newsSourceService;
    private CollectedNewsService collectedNewsService;
    private PictureDownloader pictureDownloader;
    private String imageBaseFolder;    //folder like: D:\\SrcResearch\\iFashion\\Fashion\\src\\main\\webapp\\userfiles\\pics
    private String imageContextPath;   //path like: /userfiles/pics
    private static Logger logger = Logger.getLogger(ScheduledNewsCollector.class.getName());
    private static int max_number = 300;
    private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    private static final long dayInMilliseconds = 1000 * 60 * 60 * 24;
    List<CollectedNews> collectedNewsList;

    public void collectNewsList() {
        logger.info("start to collect news");
        if (!imageContextPath.endsWith("/")) {
            imageContextPath = imageContextPath + "/";
        }

        List<NewsSource> newsSourceList = newsSourceService.findAllNewsSources();
        collectedNewsList = collectedNewsService.findAllCollectedNewsWithTitle();
        Date lastUpdate = new Date();

        for (NewsSource newsSource : newsSourceList) {
            try {
                logger.info("start to collect news from " + newsSource.getStartUrl());
                String timeStamp = StringUtils.generateNameByYearAndMonth();
                if (StringUtils.isNullOrEmpty(newsSource.getImageFolder())) {
                    newsSource.setImageFolder("default");
                }

                String pathPrefixInNews = imageContextPath + newsSource.getImageFolder() + "/" + timeStamp;
                String imageFolder = createPictureFolder(imageBaseFolder, newsSource.getImageFolder(), timeStamp);

                collectNewsListFromSource(newsSource.getStartUrl(), newsSource, imageFolder, pathPrefixInNews);
                newsSource.setLastUpdate(lastUpdate);

                newsSourceService.saveOrUpdate(newsSource);
            } catch (Exception e) {
                logger.error("Error happened in collecting news from: " + newsSource.getStartUrl(), e);
            }
        }
    }

    protected void collectNewsListFromSource(String url, NewsSource newsSource, String imageFolder, String pathPrefixInNews) throws IOException {

        boolean newsOutdated = false;
        Document document = Jsoup.parse(new URL(url), 30 * 1000);
        Elements newsLinks = document.select(newsSource.getListSelector());

        for (Element newsLink : newsLinks) {
            String newsText = newsLink.text();
            String newsHref = newsLink.attr("href");

            try {
                if (!collectNews(new URL(newsSource.getListBasePath() + newsHref), newsSource, imageFolder, pathPrefixInNews)) {
                    newsOutdated = true;
                }
            } catch (CollectException e) {
                logger.error("exception in collect news " + newsText, e);
            }
        }

        Elements nextList = document.select(newsSource.getListNext());
        if (nextList.size() > 0 && !newsOutdated) {
            logger.debug("Found next list page!");
            Element nextLink = nextList.first();
            collectNewsListFromSource(newsSource.getListBasePath() + nextLink.attr("href"), newsSource, imageFolder, pathPrefixInNews);
        } else {
            logger.debug("Stop scanning next page");
        }
    }

    protected String createPictureFolder(String imageBaseFolder, String imageFolder, String timeStamp) {
        String imageFolderPath = imageBaseFolder + File.separator + imageFolder;
        File imageFolderFile = new File(imageFolderPath);

        if (!imageFolderFile.exists()) {
            if (!imageFolderFile.mkdir()) {
                logger.error("can not create image folder: " + imageFolderPath);
            }
        }

        String imageTimeStampPath = imageFolderPath + File.separator + timeStamp;
        File imageTimeStampFile = new File(imageTimeStampPath);

        if (!imageTimeStampFile.exists()) {
            if (!imageTimeStampFile.mkdir()) {
                logger.error("can not create image time stamp folder: " + imageTimeStampPath);
            }
        }

        return imageTimeStampPath;
    }

    protected boolean collectNews(URL url, NewsSource newsSource, String imageFolder, String pathPrefixInNews) throws CollectException {
        try {
            Document document = Jsoup.parse(url, 60 * 1000);
            Elements dateElements = document.select(newsSource.getDate());
            if (dateElements.size() < 1) {
                logger.error("Cannot find date for: " + url);
                return true;
            }

            String dateString = fetchDateString(dateElements.first().text());
            String sourceDateFormat = newsSource.getDateFormat();
            SimpleDateFormat dateFormat = new SimpleDateFormat(sourceDateFormat);
            Date newsCreateDate = dateFormat.parse(dateString);
            Date lastUpdate = newsSource.getLastUpdate();
            if (isNewsOutdated(newsCreateDate, lastUpdate, sourceDateFormat)) {
                logger.debug("collectedNews outdated: " + url);

                return false;
            }

            Elements titleElements = document.select(newsSource.getTitle());
            if (titleElements.size() < 1) {
                logger.error("Cannot find title for: " + url);
                return true;
            }
            String title = titleElements.first().text();
            if (isNewsAlreadyCollected(title)) {
               return false;
            }

            CollectedNews collectedNews = new CollectedNews();
            collectedNews.setId(StringUtils.createTimeStamp());
            collectedNews.setSource(newsSource.getSource());
            collectedNews.setSourceLink(url.toString());


            collectedNews.setTitle(title);

            String introductionSelector = newsSource.getIntroduction();
            if (!StringUtils.isNullOrEmpty(introductionSelector)) {
                Elements introductionElements = document.select(introductionSelector);

                if (introductionElements.size() > 0) {
                    collectedNews.setIntroduction(introductionElements.first().text());
                }
            }

            Elements bodyElements = document.select(newsSource.getContext());

            if (bodyElements.size() < 1) {
                logger.error("Cannot find body for: " + url);
                return true;
            }

            collectedNews.setCreateDate(newsCreateDate);

            StringBuilder bodyBuilder = new StringBuilder("");
            Element bodyElement = bodyElements.first();
            downloadPictures(bodyElement, newsSource.getImageBasePath(), imageFolder, pathPrefixInNews);
            bodyBuilder.append(bodyElement.html());

            Elements nextElements = document.select(newsSource.getNext());
            if (nextElements.size() > 0) {
                bodyBuilder.append(NewsService.PageSplitter);
                traverseBody(new URL(newsSource.getNextBasePath() + nextElements.first().attr("href")), bodyBuilder, newsSource, imageFolder, pathPrefixInNews);
            }

            collectedNews.setBody(bodyBuilder.toString());
            collectedNewsService.saveOrUpdate(collectedNews);
        } catch (IOException e) {
            logger.error("Can not access: " + url);
            throw new CollectException("Can not access: " + url, e);
        } catch (ParseException e) {
            logger.error("Can not parse date for: " + url);
            throw new CollectException("Can not access: " + url, e);
        }

        return true;
    }

    protected boolean isNewsAlreadyCollected(String title) {
        if (StringUtils.isNullOrEmpty(title)) {
            return true;
        }

        for (CollectedNews collectedNews : collectedNewsList) {
            if (title.equals(collectedNews.getTitle())) {
                return true;
            }
        }

        return false;
    }

    protected boolean isNewsOutdated(Date newsCreateDate, Date lastUpdate, String sourceDateFormat) {
        if (sourceDateFormat.contains(":") || sourceDateFormat.contains("HH") || sourceDateFormat.contains("hh")) {
            //It means the there is hour in date format
            if ((lastUpdate.getTime() - newsCreateDate.getTime()) > 0) {
                return true;
            } else {
                return false;
            }
        } else {
            //there is only day in date format, then all date string will be like '2013-03-16 00:00:00'
            if ((lastUpdate.getTime() - newsCreateDate.getTime()) > dayInMilliseconds) {
                return true;
            } else {
                return false;
            }
        }
    }

    protected void traverseBody(URL url, StringBuilder bodyBuilder, NewsSource newsSource, String imageFolder, String pathPrefixInNews) throws CollectException {
        try {
            Document document = Jsoup.parse(url, 30 * 1000);
            Elements bodyElements = document.select(newsSource.getContext());
            if (bodyElements.size() < 1) {
                logger.error("Cannot find body in traverse for: " + url);
                return;
            }

            Element bodyElement = bodyElements.first();

            downloadPictures(bodyElement, newsSource.getImageBasePath(), imageFolder, pathPrefixInNews);
            bodyBuilder.append(bodyElement.html());

            Elements nextElements = document.select(newsSource.getNext());
            if (nextElements.size() > 0) {
                bodyBuilder.append(NewsService.PageSplitter);
                traverseBody(new URL(newsSource.getNextBasePath() + nextElements.first().attr("href")), bodyBuilder, newsSource, imageFolder, pathPrefixInNews);
            }
        } catch (IOException e) {
            logger.error("Can not access: " + url);
            throw new CollectException("Can not access: " + url, e);
        }
    }

    protected void downloadPictures(Element bodyElement, String basePath, String imageFolder, String pathPrefixInNews) {
        Elements imageElements = bodyElement.select("img");
        for (Element imageElement : imageElements) {
            String imageHref = imageElement.attr("src");
            String imageUrl = imageHref;

            if (!imageHref.startsWith("http")) {
                imageUrl = basePath + imageHref;
            }

            if(imageHref.indexOf(".") < 0){
                logger.error("male formatted picture url: " + imageUrl);

                return;
            }

            String fileType = imageHref.substring(imageHref.lastIndexOf("."));
            String imageName = StringUtils.createTimeStamp() + fileType;
            String imagePathInNews = pathPrefixInNews + "/" + imageName;
            String savedImageName = imageFolder + File.separator + imageName;

            pictureDownloader.saveUrlAs(imageUrl, savedImageName);
            imageElement.attr("src", imagePathInNews);
        }
    }


    protected String fetchDateString(String input) {
        return input.substring(input.indexOf("20"));
    }

    public void setPictureDownloader(PictureDownloader pictureDownloader) {
        this.pictureDownloader = pictureDownloader;
    }

    public void setNewsSourceService(NewsSourceService newsSourceService) {
        this.newsSourceService = newsSourceService;
    }

    public void setImageBaseFolder(String imageBaseFolder) {
        this.imageBaseFolder = imageBaseFolder;
    }

    public void setCollectedNewsService(CollectedNewsService collectedNewsService) {
        this.collectedNewsService = collectedNewsService;
    }

    public void setImageContextPath(String imageContextPath) {
        this.imageContextPath = imageContextPath;
    }
}
